# -*- coding: utf-8 -*-
import scrapy

import re
import json
import logging
from copy import copy, deepcopy
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem

logger = logging.getLogger(__name__)


class ZiyangSpider(scrapy.Spider):
    name = 'ziyang'
    allowed_domains = ['ziyang.gov.cn']
    start_urls = ['http://sfgj.ziyang.gov.cn/jyw/xf/newhouse_lb.html']
    project_li_url = 'http://sfgj.ziyang.gov.cn/jyw/xf/ajxfsell.aspx'  # POST 项目列表url
    project_detail_url = 'http://sfgj.ziyang.gov.cn/jyw/xf/ajxfsell.aspx'  # POST 项目详情url
    permit_li_url = 'http://sfgj.ziyang.gov.cn/jyw/xf/GetBTable.ashx'  # POST 预售许可证列表url
    room_li_url = 'http://sfgj.ziyang.gov.cn/jyw/xf/GetBTable.ashx'  # POST 房号列表url

    def start_requests(self):
        """
        获取项目列表首页
        :return:
        """
        headers = {
            'Referer': self.start_urls[0],
        }
        data = {
            'action': 'get_xfsell_list',
            'page': '1',
            'qy': '',
            'pq': '',
            'wylx': '',
            'xmjj': '',
            'keyword': '',
            'px': '',
        }
        yield scrapy.FormRequest(
            self.project_li_url,
            headers=headers,
            formdata=data,
            meta=dict(page_num=1),
        )

    def parse(self, response):
        """
        获取项目列表
        :param response:
        :return:
        """
        page_num = copy(response.meta['page_num'])
        try:
            resp_dict = json.loads(response.body.decode())
            total_num = int(resp_dict.get('total', 0))
            assert total_num, f'第{page_num}页，项目总数提取出错'
            project_li = resp_dict.get('list', list())
            assert project_li, f'第{page_num}页，项目列表提取出错'
        except AssertionError as e:
            logger.error(e)
        except Exception as e:
            logger.error(f'第{page_num}页 项目列表获取出错，error_msg:{e}')
        else:
            for project_dict in project_li:
                item_eg = FdcEstateGuidItem()
                item_eg['projectName'] = project_dict.get('XSMC', None)
                item_eg['developerName'] = project_dict.get('KFDW', None)
                item_eg['projectAddress'] = project_dict.get('XMDZ', None)
                item_eg['lngBd'] = project_dict.get('X', None)
                item_eg['latBd'] = project_dict.get('Y', None)
                item_eg['districtCode'] = project_dict.get('QY', None)
                item_eg['districtName'] = self._get_district_name(item_eg['districtCode'])
                item_eg['projectId'] = project_dict.get('DAH', None)

                # 构造POST请求，获取项目详情
                if item_eg['projectId']:
                    referer_url_temp1 = 'http://sfgj.ziyang.gov.cn/jyw/xf/newhouse_xx_xi.html?dah={}&qy={}'
                    headers = {
                        'Referer': referer_url_temp1.format(item_eg['projectId'], item_eg['districtCode']),
                    }
                    data = {
                        'action': 'get_xfsell_xxxi',
                        'dah': item_eg['projectId'],
                    }
                    yield scrapy.FormRequest(
                        self.project_detail_url,
                        headers=headers,
                        formdata=data,
                        callback=self.parse_project_detail,
                        meta=dict(item_eg=deepcopy(item_eg), ),
                        priority=6,
                    )
                else:
                    logger.error('第{}页-{} 项目id获取为空'.format(page_num, item_eg['projectName'], ))

            # 翻页
            total_page_num = total_num // 6 + 1 if total_num % 6 else total_num // 6
            referer_url_temp2 = 'http://sfgj.ziyang.gov.cn/jyw/xf/newhouse_lb.html?qy=&wylx=&xmjj=&px=&keyword=&page={}'
            for page_num in range(2, total_page_num + 1):
                headers = {
                    'Referer': referer_url_temp2.format(page_num),
                }
                data = {
                    'action': 'get_xfsell_list',
                    'page': str(page_num),
                    'qy': '',
                    'pq': '',
                    'wylx': '',
                    'xmjj': '',
                    'keyword': '',
                    'px': '',
                }
                yield scrapy.FormRequest(
                    self.project_li_url,
                    headers=headers,
                    formdata=data,
                    priority=5,
                    meta=dict(page_num=deepcopy(page_num), ),
                )

    def parse_project_detail(self, response):
        """
        获取项目详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        try:
            resp_dict = json.loads(response.body.decode())
            project_base_dict = resp_dict.get('info', None)
            assert project_base_dict, '{}/{} 项目详情字典获取为空'.format(item_eg['projectName'], item_eg['projectId'], )
            project_property_li = resp_dict.get('list', None)
        except AssertionError as e:
            logger.error(e)
        except Exception as e:
            logger.error('{}/{} 项目详情获取出错，error_msg:{}'.format(item_eg['projectName'], item_eg['projectId'], e, ))
        else:
            item_eg['parkingSpacesNum'] = project_base_dict.get('CWS', None)
            item_eg['greeningRate'] = project_base_dict.get('LHL', None)
            item_eg['floorAreaRatio'] = project_base_dict.get('RJL', None)
            item_eg['avgPrice'] = project_base_dict.get('JGXQ', None)
            item_eg['coverageArea'] = project_base_dict.get('ZDMJ', None)
            item_eg['totalArea'] = project_base_dict.get('JZMJ', None)
            item_eg['totalBuildingNum'] = project_base_dict.get('ZDS', None)
            item_eg['projectBrief'] = project_base_dict.get('XMJS', None)
            item_eg['publicName'] = project_base_dict.get('XMMC', None)
            item_eg['propertyType'] = list()
            item_eg['buildingForm'] = list()
            item_eg['decorateSituation'] = list()
            item_eg['landUsageTerm'] = list()
            item_eg['propertyCompany'] = list()
            item_eg['propertyManagementFee'] = list()
            item_eg['openingDate'] = list()
            item_eg['handoverDate'] = list()
            # 处理多个物业信息的情况
            if project_property_li:
                for project_property_dict in project_property_li:
                    item_eg['propertyType'].append(project_property_dict.get('WYLXMC', None))
                    item_eg['buildingForm'].append(project_property_dict.get('JZLXMC', None))
                    item_eg['decorateSituation'].append(project_property_dict.get('ZXBZMC', None))
                    item_eg['landUsageTerm'].append(project_property_dict.get('TDSYNX', None))
                    item_eg['propertyCompany'].append(project_property_dict.get('WYGS', None))
                    item_eg['propertyManagementFee'].append(project_property_dict.get('WYFY', None))
                    item_eg['openingDate'].append(project_property_dict.get('KPSJ', None))
                    item_eg['handoverDate'].append(project_property_dict.get('JFSJ', None))
            else:
                logger.warning('{}/{} 项目物业信息列表为空'.format(item_eg['projectName'], item_eg['projectId'], ))

            # 构造POST请求，获取预售许可证列表
            referer_url_temp = 'http://sfgj.ziyang.gov.cn/jyw/xf/newhouse_xx_lpb.html?dah={}'
            headers = {
                'Referer': referer_url_temp.format(item_eg['projectId']),
            }
            data = {
                'itemRecord': str(item_eg['projectId']),
                'houseCode': '0',
                'qy': str(item_eg['districtCode']),
            }
            yield scrapy.FormRequest(
                self.permit_li_url,
                headers=headers,
                formdata=data,
                callback=self.parse_permit_li,
                meta=dict(item_eg=deepcopy(item_eg), ),
                priority=7,
            )

    def parse_permit_li(self, response):
        """
        获取预售许可证列表
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        try:
            assert response.status == 200, '{}/{} 预售许可证列表响应状态码出错'.format(item_eg['projectName'], item_eg['projectId'])
        except AssertionError as e:
            logger.error(e)
        else:
            # 获取许可证列表分组
            tr_li = response.xpath("//tr[position()>1]")
            if tr_li:
                eg_filter_li = list()
                for tr in tr_li:
                    item_eg['preSalePermit'] = self._deal_permit_info(tr.xpath("./td[1]/text()").extract())
                    item_eg['certDate'] = self._deal_permit_info(tr.xpath("./td[4]/text()").extract())
                    if item_eg['preSalePermit'] not in eg_filter_li:
                        eg_filter_li.append(item_eg['preSalePermit'])
                        yield item_eg

                    # 实例化楼栋item
                    item_bd = FdcBuildingItem()
                    item_bd['projectName'] = item_eg['projectName']
                    item_bd['projectId'] = item_eg['projectId']
                    item_bd['preSalePermit'] = item_eg['preSalePermit']
                    item_bd['blockName'] = self._deal_building_info(tr.xpath("./td[5]/text()").extract())
                    item_bd['totalRoomNo'] = self._deal_building_info(tr.xpath("./td[2]/text()").extract())
                    item_bd['saleableRoomNo'] = self._deal_building_info(tr.xpath("./td[3]/span/text()").extract())
                    item_bd['buildingId'] = self._get_building_id(tr.xpath("./td[6]/a/@onclick").extract())

                    # 构造POST请求，获取房号列表
                    if item_bd['buildingId']:
                        referer_url_temp = 'http://sfgj.ziyang.gov.cn/jyw/xf/newhouse_xx_lpb.html?dah={}'
                        headers = {
                            'Referer': referer_url_temp.format(item_bd['projectId']),
                        }
                        data = {
                            'itemRecord': str(item_bd['projectId']),
                            'houseCode': str(item_bd['buildingId']),
                            'qy': str(item_eg['districtCode']),
                        }
                        yield scrapy.FormRequest(
                            self.room_li_url,
                            headers=headers,
                            formdata=data,
                            callback=self.parse_room_li,
                            meta=dict(item_bd=deepcopy(item_bd), ),
                            priority=8,
                        )
                    else:
                        logger.error('{}/{}-{} 楼栋id提取为空'.format(item_bd['projectName'], item_bd['projectId'],
                                                                item_bd['blockName'], ))
            else:
                logger.warning('{}/{} 预售许可证列表获取为空'.format(item_eg['projectName'], item_eg['projectId']))

    def parse_room_li(self, response):
        """
        获取房号列表
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        try:
            assert response.status == 200, '{}/{}-{}/{}  房号列表响应状态码出错'.format(item_bd['projectName'],
                                                                             item_bd['projectId'], item_bd['blockName'],
                                                                             item_bd['buildingId'], )
            # 获取楼层分组
            floor_tr = response.xpath("//tr//table//tr")
            assert floor_tr, '{}/{}-{}/{}  房号列表楼层分组获取出错'.format(item_bd['projectName'], item_bd['projectId'],
                                                                item_bd['blockName'], item_bd['buildingId'], )
        except AssertionError as e:
            logger.error(e)
        except Exception as e:
            logger.error('{}/{}-{}/{} 房号列表获取出错，error_msg:{}'.format(item_bd['projectName'], item_bd['projectId'],
                                                                    item_bd['blockName'], item_bd['buildingId'], e))
        else:
            floor_num_list = list()  # 用于楼栋总楼层获取
            unit_set = set()  # 用于楼栋单元列表获取
            for tr in floor_tr:
                # 处理楼层num
                floor_num = tr.xpath("./td[1]/text()").extract_first()
                floor_num_1, floor_num_2 = self._get_floor_num(floor_num)
                floor_num_list.append(floor_num_2) if isinstance(floor_num_2, int) else False
                # 获取同楼层下的房号分组
                room_td_li = tr.xpath("./td[position()>1]")
                if room_td_li:
                    for td in room_td_li:
                        item_rm = FdcRoomItem()
                        item_rm['projectName'] = item_bd['projectName']
                        item_rm['projectId'] = item_bd['projectId']
                        item_rm['blockName'] = item_bd['blockName']
                        item_rm['buildingId'] = item_bd['buildingId']
                        item_rm['roomFloor'] = floor_num_1
                        item_rm['saleStatus'] = self._get_sale_status(td.xpath("./@bgcolor").extract_first())
                        item_rm['roomNo'] = self._deal_room_no(td.xpath("./text()").extract_first())
                        item_rm['unitNo'] = self._get_unit_num(item_rm['roomNo'])
                        unit_set.add(item_rm['unitNo']) if item_rm['unitNo'] else False
                        td_title_str = td.xpath("./@title").extract_first()
                        room_info_dict = self._deal_room_info(td_title_str)
                        item_rm['roomUse'] = room_info_dict.get('物业类别', None)
                        item_rm['roomArea'] = room_info_dict.get('建筑面积', None)
                        item_rm['roomTotalPrice'] = room_info_dict.get('总价', None)
                        yield item_rm
                else:
                    logger.warning(
                        '{}-{}-{} 楼层房号分组获取为空'.format(item_bd['projectName'], item_bd['blockName'], floor_num_1))

            # 提取楼栋总层数
            floor_num_list.sort(reverse=True)
            item_bd['floorTotalNo'] = floor_num_list[0] if len(floor_num_list) else None
            # 添加楼栋单元列表字段
            unit_li = list(unit_set) if unit_set else ['1']  # 对于无单元的楼栋默认设置为1单元
            item_bd['unitLi'] = unit_li
            yield item_bd

    @staticmethod
    def _get_district_name(_str) -> str or None:
        """
        获取行政区
        :param _str:
        :return:
        """
        district_dict = {
            1: '城中区',
            2: '城南区',
            3: '城东区',
            4: '临空区',
        }
        try:
            ret = district_dict.get(int(_str), None)
        except:
            return
        else:
            return ret

    @staticmethod
    def _deal_permit_info(_li: list) -> str or None:
        """
        处理预售许可证信息
        :param _li:
        :return:
        """
        try:
            _str = _li[0]
            ret_li = _str.replace('：', ':').split(':', maxsplit=1)
            ret = ret_li[1]
            assert len(ret_li) == 2 and ret
        except:
            return
        else:
            return ret

    @staticmethod
    def _deal_building_info(_li: list) -> str or None:
        """
        处理楼栋信息
        :param _li: list
        :return:
        """
        try:
            ret = _li[0]
            assert ret
        except:
            return
        else:
            return ret

    @staticmethod
    def _get_building_id(_li: list) -> str or None:
        """
        提取楼栋id
        :param _str:
        :return:
        """
        regex_1 = re.compile(r'[\"\']')
        regex_2 = re.compile(r'GetData[(](.*)[)]')
        try:
            ret_str = regex_1.sub('', _li[0])
            ret_li = regex_2.findall(ret_str)[0].split(',')
            assert len(ret_li) == 3
            ret = ret_li[1]
            assert ret
        except:
            return
        else:
            return ret

    @staticmethod
    def _get_floor_num(_str) -> tuple:
        """
        提取楼层数字
        :param _str:
        :return:
        """
        regex_1 = re.compile(r'\s+')
        regex_2 = re.compile(r'-\d+|\d+')
        try:
            ret_1 = regex_1.sub('', _str)
            ret_2 = int(regex_2.findall(ret_1)[0])
        except:
            return None, None
        else:
            return ret_1, ret_2

    @staticmethod
    def _get_sale_status(_str) -> str or None:
        """
        获取房号销售状态
        :param _str:
        :return:
        """
        sale_status_dict = {
            '#06FB0A': '可售',
            '#02FFFD': '签约',
            '#FFFC06': '备案',
            '#FC7E7F': '抵押',
            '#7F0102': '查封',
            '#D7D3D3': '非售',
        }
        try:
            ret = sale_status_dict.get(_str, '其它')
        except:
            return
        else:
            return ret

    @staticmethod
    def _deal_room_no(_str) -> str or None:
        """
        处理房号num
        :param _str:
        :return:
        """
        regex = re.compile(r'\s+')
        try:
            ret = regex.sub('', _str)
        except:
            return _str
        else:
            return ret

    @staticmethod
    def _get_unit_num(_str) -> str or None:
        """
        提取单元信息
        :param _str:
        :return:
        """
        try:
            if '单元' in _str:
                unit_num = _str.split('单元')[0]
                assert unit_num
            else:
                unit_num = None
        except:
            return
        else:
            return unit_num

    @staticmethod
    def _deal_room_info(_str) -> dict:
        """
        处理房号信息，包括物业类别、建筑面积、总价等
        :param _str: 例如：物业类别：住宅  销售状态：可售  建筑面积：89.390平米  总价：62.13万元
        :return: 例如：{'物业类别': '住宅', '销售状态': '可售', '建筑面积': '89.390平米', '总价': '62.13万元'}
        """
        regex_1 = re.compile(r'\s+')
        regex_2 = re.compile(r'：')
        try:
            ret_str1 = regex_1.sub(',', _str)
            ret_str2 = regex_2.sub(':', ret_str1)
            ret_li = ret_str2.split(',')
            ret_dict = {i.split(':')[0]: i.split(':')[1] if '-' not in i.split(':')[1] else None for i in ret_li if i}
        except:
            return dict()
        else:
            return ret_dict
